{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import SVG\n",
    "from sklearn.datasets import load_digits\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Input, Dense, concatenate, Activation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset\n",
    "- digits dataset in scikit-learn\n",
    "- url: http://scikit-learn.org/stable/auto_examples/datasets/plot_digits_last_image.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_data = data.images\n",
    "y_data = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flatten X_data\n",
    "X_data = X_data.reshape(X_data.shape[0], X_data.shape[1]*X_data.shape[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1797, 64)\n",
      "(1797,)\n"
     ]
    }
   ],
   "source": [
    "# shape of data\n",
    "print(X_data.shape)\n",
    "print(y_data.shape)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequential Model API\n",
    "- Simplest way to create a deep learning model in Keras is utilizing Sequential Model API\n",
    "- Sequential Model API only enables connecting layers sequentially, hence it is **\"a linear stack of layers\"**\n",
    "    - Hence, it is effective to use when creating a \"simple\" model with a single output such as MLPs\n",
    "- doc: https://keras.io/getting-started/sequential-model-guide/\n",
    "\n",
    "<br>\n",
    "<img src=\"https://www.researchgate.net/profile/Abdelazim_Negm/publication/273768094/figure/fig2/AS:294800436809735@1447297309947/Figure-4-A-hypothetical-example-of-Multilayer-Perceptron-Network.png\" style=\"width: 400px\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Method 1\n",
    "- Passing a list of layers into Sequential()\n",
    "- Efficient to use when the model is small (i.e., shallow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential([Dense(10, input_shape = X_data.shape[1:]), Dense(10, activation = 'softmax')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 10)                650       \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                110       \n",
      "=================================================================\n",
      "Total params: 760\n",
      "Trainable params: 760\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"191pt\" viewBox=\"0.00 0.00 181.00 191.00\" width=\"181pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 187)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-187 177,-187 177,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2548645893176 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2548645893176</title>\n",
       "<polygon fill=\"none\" points=\"0,-146.5 0,-182.5 173,-182.5 173,-146.5 0,-146.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-160.8\">dense_1_input: InputLayer</text>\n",
       "</g>\n",
       "<!-- 2548648787920 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2548648787920</title>\n",
       "<polygon fill=\"none\" points=\"33,-73.5 33,-109.5 140,-109.5 140,-73.5 33,-73.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-87.8\">dense_1: Dense</text>\n",
       "</g>\n",
       "<!-- 2548645893176&#45;&gt;2548648787920 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2548645893176-&gt;2548648787920</title>\n",
       "<path d=\"M86.5,-146.313C86.5,-138.289 86.5,-128.547 86.5,-119.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-119.529 86.5,-109.529 83.0001,-119.529 90.0001,-119.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548645892672 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2548645892672</title>\n",
       "<polygon fill=\"none\" points=\"33,-0.5 33,-36.5 140,-36.5 140,-0.5 33,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-14.8\">dense_2: Dense</text>\n",
       "</g>\n",
       "<!-- 2548648787920&#45;&gt;2548645892672 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2548648787920-&gt;2548645892672</title>\n",
       "<path d=\"M86.5,-73.3129C86.5,-65.2895 86.5,-55.5475 86.5,-46.5691\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-46.5288 86.5,-36.5288 83.0001,-46.5289 90.0001,-46.5288\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Method 2\n",
    "- Sometimes, if the model gets deeper, it is difficult to get lost with list of layers\n",
    "- In this case, one can add layers line-by-line with **add()** function\n",
    "- By doing so, one can also keep track of output shape of the model step-by-step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 10)\n"
     ]
    }
   ],
   "source": [
    "model.add(Dense(10, input_shape = X_data.shape[1:], activation = 'relu'))\n",
    "print(model.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 50)\n"
     ]
    }
   ],
   "source": [
    "model.add(Dense(50, activation = 'relu'))\n",
    "model.add(Dense(50, activation = 'relu'))\n",
    "print(model.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 10)\n"
     ]
    }
   ],
   "source": [
    "model.add(Dense(10, activation = 'sigmoid'))\n",
    "print(model.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_7 (Dense)              (None, 10)                650       \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 50)                550       \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 50)                2550      \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 10)                510       \n",
      "=================================================================\n",
      "Total params: 4,260\n",
      "Trainable params: 4,260\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"337pt\" viewBox=\"0.00 0.00 181.00 337.00\" width=\"181pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 333)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-333 177,-333 177,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2548647394160 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2548647394160</title>\n",
       "<polygon fill=\"none\" points=\"0,-292.5 0,-328.5 173,-328.5 173,-292.5 0,-292.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-306.8\">dense_7_input: InputLayer</text>\n",
       "</g>\n",
       "<!-- 2548647113000 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2548647113000</title>\n",
       "<polygon fill=\"none\" points=\"33,-219.5 33,-255.5 140,-255.5 140,-219.5 33,-219.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-233.8\">dense_7: Dense</text>\n",
       "</g>\n",
       "<!-- 2548647394160&#45;&gt;2548647113000 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2548647394160-&gt;2548647113000</title>\n",
       "<path d=\"M86.5,-292.313C86.5,-284.289 86.5,-274.547 86.5,-265.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-265.529 86.5,-255.529 83.0001,-265.529 90.0001,-265.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548647452176 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2548647452176</title>\n",
       "<polygon fill=\"none\" points=\"33,-146.5 33,-182.5 140,-182.5 140,-146.5 33,-146.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-160.8\">dense_8: Dense</text>\n",
       "</g>\n",
       "<!-- 2548647113000&#45;&gt;2548647452176 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2548647113000-&gt;2548647452176</title>\n",
       "<path d=\"M86.5,-219.313C86.5,-211.289 86.5,-201.547 86.5,-192.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-192.529 86.5,-182.529 83.0001,-192.529 90.0001,-192.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548647451672 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2548647451672</title>\n",
       "<polygon fill=\"none\" points=\"33,-73.5 33,-109.5 140,-109.5 140,-73.5 33,-73.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-87.8\">dense_9: Dense</text>\n",
       "</g>\n",
       "<!-- 2548647452176&#45;&gt;2548647451672 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2548647452176-&gt;2548647451672</title>\n",
       "<path d=\"M86.5,-146.313C86.5,-138.289 86.5,-128.547 86.5,-119.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-119.529 86.5,-109.529 83.0001,-119.529 90.0001,-119.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548646439792 -->\n",
       "<g class=\"node\" id=\"node5\"><title>2548646439792</title>\n",
       "<polygon fill=\"none\" points=\"29.5,-0.5 29.5,-36.5 143.5,-36.5 143.5,-0.5 29.5,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"86.5\" y=\"-14.8\">dense_10: Dense</text>\n",
       "</g>\n",
       "<!-- 2548647451672&#45;&gt;2548646439792 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>2548647451672-&gt;2548646439792</title>\n",
       "<path d=\"M86.5,-73.3129C86.5,-65.2895 86.5,-55.5475 86.5,-46.5691\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"90.0001,-46.5288 86.5,-36.5288 83.0001,-46.5289 90.0001,-46.5288\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "- One can keep track of layer by setting their 'names'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "Input_layer (Dense)          (None, 10)                650       \n",
      "_________________________________________________________________\n",
      "First_hidden_layer (Dense)   (None, 50)                550       \n",
      "_________________________________________________________________\n",
      "Output_layer (Dense)         (None, 10)                510       \n",
      "=================================================================\n",
      "Total params: 1,710\n",
      "Trainable params: 1,710\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(10, input_shape = X_data.shape[1:], activation = 'relu', name = 'Input_layer'))\n",
    "model.add(Dense(50, activation = 'relu', name = 'First_hidden_layer'))\n",
    "model.add(Dense(10, activation = 'softmax', name = 'Output_layer'))\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"264pt\" viewBox=\"0.00 0.00 199.00 264.00\" width=\"199pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 260)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-260 195,-260 195,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 2548650433672 -->\n",
       "<g class=\"node\" id=\"node1\"><title>2548650433672</title>\n",
       "<polygon fill=\"none\" points=\"0,-219.5 0,-255.5 191,-255.5 191,-219.5 0,-219.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"95.5\" y=\"-233.8\">Input_layer_input: InputLayer</text>\n",
       "</g>\n",
       "<!-- 2548650433224 -->\n",
       "<g class=\"node\" id=\"node2\"><title>2548650433224</title>\n",
       "<polygon fill=\"none\" points=\"33,-146.5 33,-182.5 158,-182.5 158,-146.5 33,-146.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"95.5\" y=\"-160.8\">Input_layer: Dense</text>\n",
       "</g>\n",
       "<!-- 2548650433672&#45;&gt;2548650433224 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>2548650433672-&gt;2548650433224</title>\n",
       "<path d=\"M95.5,-219.313C95.5,-211.289 95.5,-201.547 95.5,-192.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"99.0001,-192.529 95.5,-182.529 92.0001,-192.529 99.0001,-192.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548650439792 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2548650439792</title>\n",
       "<polygon fill=\"none\" points=\"12,-73.5 12,-109.5 179,-109.5 179,-73.5 12,-73.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"95.5\" y=\"-87.8\">First_hidden_layer: Dense</text>\n",
       "</g>\n",
       "<!-- 2548650433224&#45;&gt;2548650439792 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>2548650433224-&gt;2548650439792</title>\n",
       "<path d=\"M95.5,-146.313C95.5,-138.289 95.5,-128.547 95.5,-119.569\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"99.0001,-119.529 95.5,-109.529 92.0001,-119.529 99.0001,-119.529\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 2548650439288 -->\n",
       "<g class=\"node\" id=\"node4\"><title>2548650439288</title>\n",
       "<polygon fill=\"none\" points=\"28,-0.5 28,-36.5 163,-36.5 163,-0.5 28,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"95.5\" y=\"-14.8\">Output_layer: Dense</text>\n",
       "</g>\n",
       "<!-- 2548650439792&#45;&gt;2548650439288 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>2548650439792-&gt;2548650439288</title>\n",
       "<path d=\"M95.5,-73.3129C95.5,-65.2895 95.5,-55.5475 95.5,-46.5691\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"99.0001,-46.5288 95.5,-36.5288 92.0001,-46.5289 99.0001,-46.5288\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVG(model_to_dot(model).create(prog='dot', format='svg'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
